<?php

namespace App\Console\Commands;

use Illuminate\Console\Command;
use Illuminate\Support\Facades\DB;
use QL\QueryList;

class Scrapy extends Command
{
    /**
     * The name and signature of the console command.
     *
     * @var string
     */
    protected $signature = 'data:scrapy';

    /**
     * The console command description.
     *
     * @var string
     */
    protected $description = '爬取数据';

    /**
     * Create a new command instance.
     *
     * @return void
     */
    public function __construct()
    {
        parent::__construct();
    }


    //趣头条
    protected function qtt()
    {
        $n = 0;
        while ($n<200) {
            $n++;
            $url = "http://api.1sapp.com/content/outList?cid=255&tn=1&page='.$n.'&limit=10&user=temporary1536300567816&show_time=&min_time=&content_type=1&dtu=200";
            $response = \Httpful\Request::get($url)
//                ->expectsJson()
                ->send();
            if(empty($response)){
                continue;
            }
            try {
                $data = $response->body->data->data;
                foreach ($data as $k=>$v) {
                    $result = [
                        'title' => isset($v->title)?$v->title:'',
                        'intro' => isset($v->introduction)?$v->introduction:'',
                        'pic' => isset($v->cover[0])?$v->cover[0]:'',
                        'cate_id' =>isset($v->type)?$v->type:'',
                        'content' => '',
                        'source_link' => isset($v->url)?$v->url:'',
                        'status' => 0,
                        'views' => mt_rand(500,2000),
                        'created_at' => date('Y-m-d H:i:s'),
                        'from_author' => isset($v->source_name)?$v->source_name:'',
                    ];
                    if($result['source_link'] == '') {
                        continue;
                    }
                    $content_url = $v->url;
                    $response = \Httpful\Request::get($content_url)
//                ->expectsJson()
                        ->send();
                    $body = $response->body;
                    preg_match('/<div class="content"><p>.+/',$body, $match_body);
                    preg_match('/<p>.+<\/p>/', $match_body[0], $body_data);
                    $body_data_ = str_replace('data-src','src',$body_data[0]);
                    $result['content'] = $body_data_;
                    //插入数据库
                    try {
                        $res = DB::table('article')->insert($result);
                        var_dump($res);
                    }catch (\Exception $e) {
                        echo "已经有数据啦\n";
                        continue;
                    }
                }
            }catch (\Exception $e) {
                echo "出错啦\n";
                continue;
//                break;
            }
        }
    }

    //东方头条
    protected function dftt()
    {
        $n = 0;
        while ($n<20) {
            $n++;
            $url = "http://pcflow.dftoutiao.com/toutiaopc_jrtt/newspool?callback=jQuery1830546233144488381_1536739001783&type=yule&uid=15366559003909331&position=%25E9%2595%25BF%25E6%25B2%2599&domain=mini&startkey=%7C%7C%7C%7C&newkey=%7C&pgnum=".$n."&idx=6&sclog=1&_=1536739081159";
            $response = \Httpful\Request::get($url)
                //->expectsJson()
                ->send();
            if(empty($response)){
                continue;
            }
            try {
                $data = $response->body;
                preg_match('/{.*}/',$data, $data_math);
                $data_arr = json_decode($data_math[0], 1);
                $data = $data_arr['data'];
                foreach ($data as $k=>$v) {
                    $result = [
                        'title' => isset($v['topic'])?$v['topic']:'',
                        'intro' => isset($v['topic'])?$v['topic']:'',
                        'pic' => '',
                        'cate_id' => 1,
                        'content' => '',
                        'source_link' => isset($v['url'])?$v['url']:'',
                        'status' => 0,
                        'views' => mt_rand(500,2000),
                        'created_at' => date('Y-m-d H:i:s'),
                        'from_author' => isset($v['source'])?$v['source']:'',
                    ];
                    if(count($v['miniimg']) > 0) {
                        foreach($v['miniimg'] as $k_m=>$v_m) {
                            $result['pic'] .= 'https:'.$v_m['src'].',';
                        }
                        $result['pic'] = substr($result['pic'],0,-1);
                    }
                    if($result['source_link'] == '') {
                        continue;
                    }
                    $content_url = $v['url'];
                    $response = \Httpful\Request::get($content_url)
//                ->expectsJson()
                        ->send();
                    $body = $response->body;
                    $ql = QueryList::html($body);
                    $content = $ql->find('.article')->find('#J-contain_detail_cnt')->htmls()->all();
                    $result['content'] = $content[0];
                    //获取分页
                    $pages = $ql->find('.pagination>a')->attrs('href');
                    $pages_arr = $pages->all();
                    $pages_arr = array_filter($pages_arr);
                    $pages_arr = array_flip(array_flip($pages_arr));
                    $pages_arr = array_values($pages_arr);
                    if(count($pages_arr) >=2 ) {
                        array_pop($pages_arr);
                    }
                    foreach ($pages_arr as $k_p=>$v_p) {
//                        $url_parse = parse_url($content_url);
//                        $p_url = $url['scheme'].'://'.$url['host'];
                        $p_url = 'https://mini.eastday.com/a/'.$v_p;
                        $response = \Httpful\Request::get($p_url)
//                ->expectsJson()
                            ->send();
                        $data = $response->body;
                        $ql = QueryList::html($data);
                        $content = $ql->find('.article')->find('#J-contain_detail_cnt')->htmls()->all();
                        $result['content'] .= $content[0];
                    }
                    //插入数据库
                    try {
                        $res = DB::table('article')->insert($result);
                        var_dump($res);
                    }catch (\Exception $e) {
                        echo "已经有数据啦\n";
                        continue;
                    }
                }
            }catch (\Exception $e) {
                echo "出错啦\n";
                continue;
//                break;
            }
        }

    }

    /**
     * Execute the console command.
     *
     * @return mixed
     */
    public function handle()
    {
        $this->dftt();
    }

    protected function requestUrl($URL)
    {
        $c = curl_init();
        curl_setopt($c, CURLOPT_RETURNTRANSFER, 1);
        //curl_setopt($c, CURLOPT_HEADER, 1);//输出远程服务器的header信息
        curl_setopt($c, CURLOPT_USERAGENT, 'User-Agent:Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36 MicroMessenger/6.5.2.501 NetType/WIFI WindowsWechat');
        curl_setopt($c, CURLOPT_URL, $URL);
        $contents = curl_exec($c);
        curl_close($c);
        if ($contents) {
            return $contents;
        } else {
            return FALSE;
        }
    }


}
